# Let's add the checkpoint for the following example:
# 3 GX000-31-12362252 employment 2.461117 2437458 3 136 7.664865 5 536 True 57844  54 10058969 0 0 0 0 0 0 0 0 0 0 2 0 0 0 0 3  14 3 0 0 0 3 5 0 0

# Re-compute the 25 features using different ranges values
# The 25 features which prof specify 
BIG_NUMBER = 1000000
# current version
Y_RANGE_1_UPPER_BOUND_for_GENERAL_POSTING_PROBABILITY = 2.90861 / BIG_NUMBER
Y_RANGE_2_UPPER_BOUND_for_GENERAL_POSTING_PROBABILITY = 11.5781 / BIG_NUMBER
Y_RANGE_3_UPPER_BOUND_for_GENERAL_POSTING_PROBABILITY = 37.4255 / BIG_NUMBER
Y_RANGE_4_UPPER_BOUND_for_GENERAL_POSTING_PROBABILITY = 125.379 / BIG_NUMBER

print "BIG_NUMBER:",BIG_NUMBER
print "Y_RANGE_1_UPPER_BOUND_for_GENERAL_POSTING_PROBABILITY:",Y_RANGE_1_UPPER_BOUND_for_GENERAL_POSTING_PROBABILITY
print "Y_RANGE_2_UPPER_BOUND_for_GENERAL_POSTING_PROBABILITY:",Y_RANGE_2_UPPER_BOUND_for_GENERAL_POSTING_PROBABILITY
print "Y_RANGE_3_UPPER_BOUND_for_GENERAL_POSTING_PROBABILITY:",Y_RANGE_3_UPPER_BOUND_for_GENERAL_POSTING_PROBABILITY
print "Y_RANGE_4_UPPER_BOUND_for_GENERAL_POSTING_PROBABILITY:",Y_RANGE_4_UPPER_BOUND_for_GENERAL_POSTING_PROBABILITY

# row1
f11 = 0
f21 = 0
f31 = 0
f41 = 0
f51 = 0

# row2
f12 = 0
f22 = 0
f32 = 0
f42 = 0
f52 = 0

# row3
f13 = 0
f23 = 0
f33 = 0
f43 = 0
f53 = 0

# row4
f14 = 0
f24 = 0
f34 = 0
f44 = 0
f54 = 0

# row5
f15 = 0
f25 = 0
f35 = 0
f45 = 0
f55 = 0

# option1(NOT used anymore):
# naive ranges dividing based on X axis and Y axis freq
# computation_method = 1

# option2(currently using):
# ranges dividing based on the histogram info of the X axis and Y axis
# computation_method = 2
computation_method = 2 

# load the externalID and internalID into main memory
print "step1:load the basic training files into a data structure called: trecIDAndInternalIDDict"
print "key:trecID and value:internalDocID"
print "example: GX000-31-12362252 681"
# key: trecID
# value: internalDocID
trecIDAndInternalIDDict = {}

inputAuxFileName0 = "/data3/obukai/workspace/web-search-engine-wei/polyIRIndexer/small_set_of_gov2_documents_needed_to_be_extracted_for_training_queries_sorted_by_trecID"
inputAuxFileHandler0 = open(inputAuxFileName0,"r")

# Example format of the line:
# 3 5 7.664865 681 GX000-31-12362252
# 0 1 2        3   4
for line in inputAuxFileHandler0.readlines():
    lineElements = line.strip().split(" ")
    internalDocID = lineElements[3]
    trecID = lineElements[4]
    if trecID not in trecIDAndInternalIDDict:
        trecIDAndInternalIDDict[trecID] = internalDocID
    else:
        print "system error"
        exit(1)

print "len(trecIDAndInternalIDDict):",len(trecIDAndInternalIDDict)
print 

# load the term_pair dict with id into main memory.
# load the term_pair ID and real content mapping into main memory
print "step2:load the (1)query term pair (2)unique ID (3)freq"
print "key:term_pair value:tuple(uniqueID,term_pair_freq)"
print "example: 0001_3064 (0,1)"
# key: term_pair
# value: tuple(uniqueID,term_pair_freq)
termPairUniqueIDWithRealContentFreqTuple = {}
inputAuxFileName2 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/queryTermPairProbabilityOfTheQueryTrace_sorted_by_query_term_pair_sorted_by_uniqueID"
inputAuxFileHandler2 = open(inputAuxFileName2,"r")

# Example format of the line:
# uniqueIDForTheTermPair theTermPair theTermPairFreq
# 0                      0001_3064   1
for line in inputAuxFileHandler2.readlines():
    lineElements = line.strip().split(" ")
    uniqueID = lineElements[0]
    term_pair = lineElements[1]
    term_pair_freq = int(lineElements[2])
    if uniqueID not in termPairUniqueIDWithRealContentFreqTuple:
        termPairUniqueIDWithRealContentFreqTuple[term_pair] = (uniqueID,term_pair_freq)

print "len(termPairUniqueIDWithRealContentFreqTuple):",len(termPairUniqueIDWithRealContentFreqTuple)
print 

# load the edges ID into main memory
print "step3:load the ConnectedEdges for each specific document"
print "key:docID value:[a list of integers representing the connected edges]"
print "example: 681 [18115 20797 22489 22521 ...]"
docIDWithTheirAssociatedEdgesInListStructure = {}
docIDWithTheirAssociatedEdgesInDictStructure = {}

inputAuxFileName1 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/gov2DocumentWithTheirConnectedEdges"
inputAuxFileHandler1 = open(inputAuxFileName1,"r")

# ignore the head line
inputAuxFileHandler1.readline()

# Example format of the line:
# trecID threshold(1) numOfEdges edgeIDs
# GX000-31-12362252 N/A 713 [18115 20797 22489 22521 22523 ...]
for line in inputAuxFileHandler1.readlines():
    lineElements = line.strip().split(" ")
    trecID = lineElements[0]
    numOfEdges = int( lineElements[1] )
    
    if trecID not in docIDWithTheirAssociatedEdgesInListStructure:
        docIDWithTheirAssociatedEdgesInListStructure[trecID] = []
        docIDWithTheirAssociatedEdgesInDictStructure[trecID] = {}
        for i in range(2,numOfEdges+2):
            docIDWithTheirAssociatedEdgesInListStructure[trecID].append( lineElements[i] )
            if lineElements[i] not in docIDWithTheirAssociatedEdgesInDictStructure[trecID]:
                docIDWithTheirAssociatedEdgesInDictStructure[trecID][ lineElements[i] ] = 1
    else:
        print "system error"
        exit(1)
    
    # for debug ONLY
    # print "numOfEdges:",numOfEdges
    # print "len( docIDWithTheirAssociatedEdgesInListStructure[trecID] ):",len( docIDWithTheirAssociatedEdgesInListStructure[trecID] )

# print "docIDWithTheirAssociatedEdgesInListStructure['681']:",docIDWithTheirAssociatedEdgesInListStructure['681']
# traversal this data structure a little bit
'''
for trecID in docIDWithTheirAssociatedEdgesInListStructure:
    print trecID,"List:",len( docIDWithTheirAssociatedEdgesInListStructure[trecID] )
    print trecID,"Dict:",len( docIDWithTheirAssociatedEdgesInDictStructure[trecID] )
'''

print "len(docIDWithTheirAssociatedEdgesInListStructure):",len(docIDWithTheirAssociatedEdgesInListStructure)
print "len(docIDWithTheirAssociatedEdgesInDictStructure):",len(docIDWithTheirAssociatedEdgesInDictStructure)
print 

print "step4:load the query term with their second parts query terms"
print "key:queryTerm value:{} another dict storing all the seconds parts of query terms for the key query term"
print "example 0001 {'3064':1,'omb':1}"
queryTermWithTheirSecondPartsDict = {}
inputAuxFileName2 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/queryTermWithTheirSecondPartsFor100KQueries"
inputAuxFileHandler2 = open(inputAuxFileName2,"r")

# Example format of the line:
# termPairFirstPart numOfSecondPart termPairSecondParts
# 000sites 7 13 high injuries list numbers of worker
for line in inputAuxFileHandler2.readlines():
    lineElements = line.strip().split(" ")
    queryTerm = lineElements[0]
    NumOfQueryTermForSecondParts = int( lineElements[1] )
    if queryTerm not in queryTermWithTheirSecondPartsDict:
        queryTermWithTheirSecondPartsDict[queryTerm] = {}
        for i in range(2,NumOfQueryTermForSecondParts+2):
            if lineElements[i] not in queryTermWithTheirSecondPartsDict[queryTerm]:
                queryTermWithTheirSecondPartsDict[queryTerm][ lineElements[i] ] = 1
            else:
                print "Duplicated"
                queryTermWithTheirSecondPartsDict[queryTerm][ lineElements[i] ] += 1

print "len(queryTermWithTheirSecondPartsDict):",len(queryTermWithTheirSecondPartsDict)
print 


print "step5:load the query term with their freq into main memory"
print "key:queryTerm value: the freq of this query term"
print "example 0001 1"
queryTermFreqDict = {}
inputAuxFileName3 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/queryTermFreqDistribution_sorted_by_queryTerm"
inputAuxFileHandler3 = open(inputAuxFileName3,"r")

# Example format of the line:
# term termFreq
# 0001 1
for line in inputAuxFileHandler3.readlines():
    lineElements = line.strip().split(" ")
    queryTerm = lineElements[0]
    queryTermFreq = int( lineElements[1] )

    if queryTerm not in queryTermFreqDict:
        queryTermFreqDict[queryTerm] = queryTermFreq

inputAuxFileHandler3.close()
print "len(queryTermFreqDict):",len(queryTermFreqDict)
print


print "step6:load the designated X range with their query term key into main memory"
print "key:queryTermPair value: the X range"
print "original line info example:0 0001_3064 1 1"
print "example 0001_3064 1"
queryTermPairWithTheirXRanges = {}
inputAuxFileName4 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/queryTermPairProbabilityOfTheQueryTrace_sorted_by_query_term_pair_sorted_by_term_pair_freq_with_designated_X_ranges_added"
inputAuxFileHandler4 = open(inputAuxFileName4,"r")

# Example format of the line:
# term termFreq
# 0001 1
for line in inputAuxFileHandler4.readlines():
    lineElements = line.strip().split(" ")
    queryTermPair = lineElements[1]
    queryTermPairXRange = int( lineElements[3] )

    if queryTermPair not in queryTermPairWithTheirXRanges:
        queryTermPairWithTheirXRanges[queryTermPair] = queryTermPairXRange

inputAuxFileHandler4.close()
print "len(queryTermPairWithTheirXRanges):",len(queryTermPairWithTheirXRanges)
print


print "Updated by Wei 2013/04/02. I am NOT going to load the information about their Y ranges with their query term key"

'''
print "Updated by Wei 2013/04/02. DO NOT NEED TO LOAD THIS DATASTRUCTURE"
print "step7:load the designated Y range with their query term key into main memory"
print "key:the probability in string format value: the Y range"
print "original line info example:2.09164e-06 1 1"
print "example 2.09164e-06 1"
probabilityWithTheirYRanges = {}
inputAuxFileName5 = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/outputDirForTermScores/histogram_all_1000000TimesProbability_sorted_by_first_column_probability_increasing_with_designated_Y_ranges_added"
inputAuxFileHandler5 = open(inputAuxFileName5,"r")

# Example format of the line:
# term termFreq
# 0001 1
for line in inputAuxFileHandler5.readlines():
    lineElements = line.strip().split(" ")
    probability = float( lineElements[0] )
    probabilityYRange = int( lineElements[2] )

    if probability not in probabilityWithTheirYRanges:
        probabilityWithTheirYRanges[probability] = probabilityYRange

inputAuxFileHandler5.close()
print "len(probabilityWithTheirYRanges):",len(probabilityWithTheirYRanges)
print
'''

print "step8:load the forward index into the main memory using the variable called: forwardIndexInMainMemory"
# key: trecID value: dict 
    # subKey: queryTerm subValue: a tuple with the following format(4.45178,718986,11.9503)
    # the meaning of the tuple is (4.45178,718986,11.9503)
    # 4.45178: BM25
    # 718986: postingProbabilityGivenTheQueryTimesBigNum1000000(I am NOT sure what is the value of the big number???)
    # 11.9503: postingProbabilityInGeneralTimesBigNum1000000(I am NOT sure what is the value of the big number???)
forwardIndexInMainMemory = {}

inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/gov2DocumentWithTheirForwardIndex"
inputFileHandler = open(inputFileName,"r")

# This is the headline info
headLine = inputFileHandler.readline()

dataLine = inputFileHandler.readline()
while dataLine:
    data = dataLine.strip()
    dataElements = data.split(" ")
    trecID = dataElements[0]
    docSizeInWords = int( dataElements[1] ) # This field will be ignored
    docPostingsRecorded = int( dataElements[2] )
    
    if trecID not in forwardIndexInMainMemory:
        forwardIndexInMainMemory[trecID] = {}
    else:
        # the word dict for the current document has been build.
        # so there is NO logic for that
        pass
    
    for i in range(0+3,docPostingsRecorded+3):
        tupleElements = dataElements[i].split("(")[1].split(")")[0].split(",")
        # the tuple element will have the following forms:
        # ['www', '0.435614', '262071', '3.73365']
        # subKey: tupleElements[0]
        # subValue: (0.435614,262071,3.73365)
        if tupleElements[0] not in forwardIndexInMainMemory[trecID]:
            valueTuple = (tupleElements[1],tupleElements[2],tupleElements[3])
            forwardIndexInMainMemory[trecID][ tupleElements[0] ] = valueTuple
    # for debug
    # print "forwardIndexInMainMemory:",forwardIndexInMainMemory
    # print "len(forwardIndexInMainMemory):",len(forwardIndexInMainMemory)
    dataLine = inputFileHandler.readline()
inputFileHandler.close()
print "len(forwardIndexInMainMemory):",len(forwardIndexInMainMemory) 



print "**********The above are all the aux files needed to support the extra features**********"
print "**********Let's generate the extra features for this small while typical training sets**********"
# exit(1)

inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/Posting_Oriented_Training_Dataset_2013_02_22.train_tfqAdded_labelsAdded_sortedByTerm_rankInListAdded_sortedByDocID_rankInDocAdded_freqInCollectionAdded_sortedByQueryID_fixed"
inputFileHandler = open(inputFileName,"r")

outputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/Posting_Oriented_Training_Dataset_2013_02_22.train_tfqAdded_labelsAdded_sortedByTerm_rankInListAdded_sortedByDocID_rankInDocAdded_freqInCollectionAdded_sortedByQueryID_fixed_extraFeaturesAdded_for_debug"
outputFileHandler = open(outputFileName,"w")

# ignore the header line
oldHeaderLine = inputFileHandler.readline()
newHeaderLine = str( oldHeaderLine.strip() + " " + "f11" + " " + "f21" + " " + "f31" + " " + "f41" + " " + "f51" + " " + 
                                                   "f12" + " " + "f22" + " " + "f32" + " " + "f42" + " " + "f52" + " " + 
                                                   "f13" + " " + "f23" + " " + "f33" + " " + "f43" + " " + "f53" + " " +
                                                   "f14" + " " + "f24" + " " + "f34" + " " + "f44" + " " + "f54" + " " +
                                                   "f15" + " " + "f25" + " " + "f35" + " " + "f45" + " " + "f55" + "\n") 
outputFileHandler.write(newHeaderLine)

for index,line in enumerate( inputFileHandler.readlines() ):
    print "old line:",index,line.strip()
    lineElements = line.strip().split(" ")
    qid = lineElements[0]
    trecID = lineElements[1]
    queryTerm = lineElements[2]
    
    # print docIDWithTheirAssociatedEdgesInListStructure[ trecIDAndInternalIDDict[trecID] ]
    # print "len(docIDWithTheirAssociatedEdgesInListStructure[ trecIDAndInternalIDDict[trecID] ]):",len(docIDWithTheirAssociatedEdgesInListStructure[ trecIDAndInternalIDDict[trecID] ])
    
    if queryTerm not in queryTermWithTheirSecondPartsDict:
        print "system error"
        exit(1)
    else:
        
        # do the normal logic
        for currentSecondPartQueryTerm in queryTermWithTheirSecondPartsDict[queryTerm]:
            if queryTerm != currentSecondPartQueryTerm:
                
                # because the query term pair is stored in the Lexicographical order, so I need compare the order of those two terms
                if queryTerm < currentSecondPartQueryTerm:
                    key = queryTerm + "_" + currentSecondPartQueryTerm
                else:
                    key = currentSecondPartQueryTerm + "_" + queryTerm
                    
                # transfer the key into an integer id number first
                
                # for debug ONLY
                # print "key:",key
                if key not in termPairUniqueIDWithRealContentFreqTuple:
                    print "unexpected behaviour"
                    exit(1)
                else:
                    ( uniqueID,freqOfTermPair ) = termPairUniqueIDWithRealContentFreqTuple[key]
                    # check whether this term pair is in the specific document identified by the internal docID
                    # for debugging purposes.
                    '''
                    print "uniqueID:",uniqueID,type(uniqueID)
                    print "trecID:",trecID,type(trecID)
                    print "freqOfTermPair:",freqOfTermPair
                    print "trecIDAndInternalIDDict[trecID]:",trecIDAndInternalIDDict[trecID],type(trecIDAndInternalIDDict[trecID])
                    '''
                    
                    if uniqueID not in docIDWithTheirAssociatedEdgesInDictStructure[ trecID ]:
                        # cause this uniqueID and the associated representing edge is NOT
                        # for debug ONLY
                        # print "uniqueID:",uniqueID,"key:",key,"is NOT in the current document"
                        pass
                    else:
                        # Let's position the X-Axis range and the Y-Axis range
                    
                        # First of all, this key pair should appear in the current document
                        # And then we can consider the X-Axis and the Y-Axis thing
                        # Lack this part of logic here
                        
                        '''
                        # option1
                        # computation_method = 1                  
                        The 5 ranges are for the X-Axis(Option1, OLD version, this can NOT be done):
                                    Range1: The temrPairFreq of term pairs appeared from ONLY once.
                                    Range2: The temrPairFreq of term pairs appeared from 2 - 10
                                    Range3: The temrPairFreq of term pairs appeared from 11 - 100
                                    Range4: The temrPairFreq of term pairs appeared from 101 - 1000
                                    Range5: The temrPairFreq of term pairs appeared from 1001 - end
                        
                        The 5 ranges are for the Y-Axis(Option1, OLD version, this can NOT be done):
                                    Range1: The secondPartQueryTermFreq of this term appeared from ONLY once.
                                    Range2: The secondPartQueryTermFreq of this term appeared from 2 - 10
                                    Range3: The secondPartQueryTermFreq of this term appeared from 11 - 100
                                    Range4: The secondPartQueryTermFreq of this term appeared from 101 - 1000
                                    Range5: The secondPartQueryTermFreq of this term appeared from 1001 - end
                        
                        # option2
                        # computation_method = 2
                        The 5 ranges are for the X-Axis(Option2, CURRENT version):
                                    Range1: This term pair appeared between [beginning, index: 165837 line: 284396 gaining_organic 1)
                                    Range2: This term pair appeared between [index: 165837 line: 284396 gaining_organic 1, index: 331674 line: 54274 aluminum_price 1)
                                    Range3: This term pair appeared between [index: 331674 line: 54274 aluminum_price 1, index: 429562 line: 39112 address_security 3)
                                    Range4: This term pair appeared between [index: 429562 line: 39112 address_security 3, index: 458179 line: 290216 global_of 15)
                                    Range5: This term pair appeared between [index: 458179 line: 290216 global_of 15, ending]

                        The 5 ranges are for the Y-Axis(Option2, CURRENT version):
                                    Range1: The predicted probability of this posting appeared between [beginning, index: 3082376 line: 2.90861 1781)
                                    Range2: The predicted probability of this posting appeared between [index: 3082376 line: 2.90861 1781, index: 3807296 line: 11.5781 11093)
                                    Range3: The predicted probability of this posting appeared between [index: 3807296 line: 11.5781 11093, index: 4065770 line: 37.4255 8575)
                                    Range4: The predicted probability of this posting appeared between [index: 4065770 line: 37.4255 8575, index: 4716894 line: 125.379 4714)
                                    Range5: The predicted probability of this posting appeared between [index: 4716894 line: 125.379 4714, ending]
                        '''
                        
                        # The logic for assigning the temrPairFreq
                        xRange = -1
                        if key not in termPairUniqueIDWithRealContentFreqTuple:
                            print "error, mark1"
                            print "key:",key
                            exit(1)
                        else:
                            (uniqueID, temrPairFreq) = termPairUniqueIDWithRealContentFreqTuple[key]
                            
                            # for debug ONLY
                            '''
                            print "designing the ranges of the X-Axis"
                            print "key:",key
                            print "temrPairFreq:",temrPairFreq
                            '''
                            # the format of some useful info
                            # key, example: gaining_organic
                            # uniqueID, for each term pair will have a uniqueID
                            # temrPairFreq, for each term pair will associate with a freq number
                            
                            if computation_method == 1:
                                # (current assigning method)
                                if temrPairFreq == 1:
                                    xRange = 1
                                elif temrPairFreq <= 10 and temrPairFreq >= 2:
                                    xRange = 2
                                elif temrPairFreq <= 100 and temrPairFreq >= 11:
                                    xRange = 3
                                elif temrPairFreq <= 1000 and temrPairFreq >= 101:
                                    xRange = 4
                                elif temrPairFreq >= 1001:
                                    xRange = 5
                            elif computation_method == 2:
                                # Note: can not use the uniqueID as the comparison operator 
                                # Note: can not use the termPair itself as the comparison operator
                                xRange = queryTermPairWithTheirXRanges[key]
                                
                        
                        
                        # The logic for assigning the secondPartQueryTermFreq
                        yRange = -1
                        if currentSecondPartQueryTerm not in queryTermFreqDict:
                            print "error, mark2"
                            exit(1)
                        else:
                            secondPartQueryTermFreq  = queryTermFreqDict[currentSecondPartQueryTerm]
                            
                            # for debug ONLY
                            '''
                            print "designing the ranges of the Y-Axis"
                            print "currentSecondPartQueryTerm:",currentSecondPartQueryTerm
                            print "secondPartQueryTermFreq:",secondPartQueryTermFreq
                            '''
                            
                            if computation_method == 1:                  
                                if secondPartQueryTermFreq == 1:
                                    yRange = 1
                                elif secondPartQueryTermFreq <= 10 and secondPartQueryTermFreq >= 2:
                                    yRange = 2
                                elif secondPartQueryTermFreq <= 100 and secondPartQueryTermFreq >= 11:
                                    yRange = 3
                                elif secondPartQueryTermFreq <= 1000 and secondPartQueryTermFreq >= 101:
                                    yRange = 4
                                elif secondPartQueryTermFreq >= 1001:
                                    yRange = 5
                            elif computation_method == 2:
                                # (0.435614,262071,3.73365) 
                                (BM25Score, postingProbabilityGivenQueryTraceInStringFormat, postingProbabilityInGeneralInStringFormat) = forwardIndexInMainMemory[trecID][currentSecondPartQueryTerm]
                                postingProbabilityInGeneral = float(postingProbabilityInGeneralInStringFormat) / BIG_NUMBER
                                

                                
                                if postingProbabilityInGeneral < Y_RANGE_1_UPPER_BOUND_for_GENERAL_POSTING_PROBABILITY:
                                    yRange = 1
                                elif postingProbabilityInGeneral < Y_RANGE_2_UPPER_BOUND_for_GENERAL_POSTING_PROBABILITY and postingProbabilityInGeneral >= Y_RANGE_1_UPPER_BOUND_for_GENERAL_POSTING_PROBABILITY:
                                    yRange = 2
                                elif postingProbabilityInGeneral < Y_RANGE_3_UPPER_BOUND_for_GENERAL_POSTING_PROBABILITY and postingProbabilityInGeneral >= Y_RANGE_2_UPPER_BOUND_for_GENERAL_POSTING_PROBABILITY:
                                    yRange = 3
                                elif postingProbabilityInGeneral < Y_RANGE_4_UPPER_BOUND_for_GENERAL_POSTING_PROBABILITY and postingProbabilityInGeneral >= Y_RANGE_3_UPPER_BOUND_for_GENERAL_POSTING_PROBABILITY:
                                    yRange = 4
                                elif postingProbabilityInGeneral >= Y_RANGE_4_UPPER_BOUND_for_GENERAL_POSTING_PROBABILITY:
                                    yRange = 5                                
                                
                                '''
                                # old version(never used)
                                yRange = probabilityWithTheirYRanges[postingProbabilityInGeneral]
                                '''
                                
                                # print "UNDER construction(2013/04/01's work)"
                                # print "mark2"
                                # pass
                                # exit(1)                                
                        
                        # for debug ONLY
                        '''
                        print "X",key,xRange
                        print "Y",trecID,currentSecondPartQueryTerm,postingProbabilityInGeneral,yRange
                        print 
                        '''
                        
                        '''
                        print "*****Update the 25 features begins..."
                        '''
                        
                        # row1
                        if xRange == 1 and yRange == 1:
                            f11 += 1
                        elif xRange == 2 and yRange == 1:
                            f21 += 1
                        elif xRange == 3 and yRange == 1:
                            f31 += 1
                        elif xRange == 4 and yRange == 1:
                            f41 += 1
                        elif xRange == 5 and yRange == 1:
                            f51 += 1
                        # row2
                        elif xRange == 1 and yRange == 2:
                            f12 += 1
                        elif xRange == 2 and yRange == 2:
                            f22 += 1
                        elif xRange == 3 and yRange == 2:
                            f32 += 1
                        elif xRange == 4 and yRange == 2:
                            f42 += 1
                        elif xRange == 5 and yRange == 2:
                            f52 += 1
                        # row3
                        elif xRange == 1 and yRange == 3:
                            f13 += 1
                        elif xRange == 2 and yRange == 3:
                            f23 += 0
                        elif xRange == 3 and yRange == 3:
                            f33 += 0
                        elif xRange == 4 and yRange == 3:
                            f43 += 0
                        elif xRange == 5 and yRange == 3:
                            f53 += 0
                        # row4
                        elif xRange == 1 and yRange == 4:
                            f14 += 1
                        elif xRange == 2 and yRange == 4:
                            f24 += 1
                        elif xRange == 3 and yRange == 4:
                            f34 += 1
                        elif xRange == 4 and yRange == 4:
                            f44 += 1
                        elif xRange == 5 and yRange == 4:
                            f54 += 1
                        # row5
                        elif xRange == 1 and yRange == 5:
                            f15 += 1
                        elif xRange == 2 and yRange == 5:
                            f25 += 1
                        elif xRange == 3 and yRange == 5:
                            f35 += 1
                        elif xRange == 4 and yRange == 5:
                            f45 += 1
                        elif xRange == 5 and yRange == 5:
                            f55 += 1
                        
                        # for debug ONLY
                        '''
                        print "*****Update the 25 features ends."
                        print
                        '''
            else:
                # Cause in this case, queryTerm == currentSecondPartQueryTerm
                pass 
        
        # for debug ONLY
        # print len(queryTermWithTheirSecondPartsDict[queryTerm])
        
        # for debug ONLY
        '''
        print "25 extra features are here:"
        # row1
        print "f11:",f11
        print "f21:",f21
        print "f31:",f31
        print "f41:",f41
        print "f51:",f51
        
        # row2
        print "f12:",f12
        print "f22:",f22
        print "f32:",f32
        print "f42:",f42
        print "f52:",f52
        
        # row3
        print "f13:",f13
        print "f23:",f23
        print "f33:",f33
        print "f43:",f43
        print "f53:",f53
        
        # row4
        print "f14:",f14
        print "f24:",f24
        print "f34:",f34
        print "f44:",f44
        print "f54:",f54
        
        # row5
        print "f15:",f15
        print "f25:",f25
        print "f35:",f35
        print "f45:",f45
        print "f55:",f55
        '''
    newOutputLine =         str(line.strip() + " " + 
                            str(f11) + " " + str(f21) + " " + str(f31) + " " + str(f41) + " " + str(f51) + " " + 
                            str(f12) + " " + str(f22) + " " + str(f32) + " " + str(f42) + " " + str(f52) + " " + 
                            str(f13) + " " + str(f23) + " " + str(f33) + " " + str(f43) + " " + str(f53) + " " +
                            str(f14) + " " + str(f24) + " " + str(f34) + " " + str(f44) + " " + str(f54) + " " +
                            str(f15) + " " + str(f25) + " " + str(f35) + " " + str(f45) + " " + str(f55) + " " + "\n")
    
    print "new line:",index,newOutputLine.strip()
    outputFileHandler.write(newOutputLine)
    
    # row1
    f11 = 0
    f21 = 0
    f31 = 0
    f41 = 0
    f51 = 0
    
    # row2
    f12 = 0
    f22 = 0
    f32 = 0
    f42 = 0
    f52 = 0
    
    # row3
    f13 = 0
    f23 = 0
    f33 = 0
    f43 = 0
    f53 = 0
    
    # row4
    f14 = 0
    f24 = 0
    f34 = 0
    f44 = 0
    f54 = 0
    
    # row5
    f15 = 0
    f25 = 0
    f35 = 0
    f45 = 0
    f55 = 0

    # for debug ONLY
    # exit(1)

inputFileHandler.close()
outputFileHandler.close()

inputAuxFileHandler0.close()
inputAuxFileHandler1.close()